package org.hagen.mmia.yars.engine;

import java.io.File;
import java.io.IOException;
import java.nio.file.DirectoryStream;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;

import javax.xml.parsers.ParserConfigurationException;

import org.xml.sax.SAXException;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import org.hagen.mmia.yars.engine.corpora.Abstract;
import org.hagen.mmia.yars.engine.corpora.ClefIP;
import org.hagen.mmia.yars.engine.corpora.ClefIPHandler;
import org.hagen.mmia.yars.engine.corpora.OhsuMed;
import org.hagen.mmia.yars.engine.corpora.Reader;
import org.hagen.mmia.yars.engine.corpora.Stop;

import org.hagen.mmia.yars.collection.Collection;
import org.hagen.mmia.yars.collection.Topic;

public class Index {

	private static boolean verbose		= false;

	// which test collection...
	public enum CollectionSize { LARGE, HUGE };
	
	// index fields
	final public static String FIELD_ID 		= "id";
	final public static String FIELD_INDEX 		= "index";
	final public static String FIELD_CONTENT 	= "content";
	final public static String FIELD_TITLE 		= "title";
	
	// index statistics
	public static double avgDocLen 	= 0.0;
	public static int	 numDocs 	= 0;

	public static Directory			  index 	= null;
	public static StandardAnalyzer	  analyzer  = null;
	
	// singleton
	private static IndexReader __reader = null;
	
	/*
	 * 
	 */
	public static IndexReader getReader() {
		
		if( __reader == null ) {
			
			try {
				__reader = DirectoryReader.open(Index.index);
			} catch (IOException e) {
				return null;
			}
		}

		return __reader;
	}

	/*
	 * 
	 */
	private static void readTopics( CollectionSize size ) throws ParserConfigurationException, SAXException {

		System.out.println("reading stop words ...");
		Stop.readWords();
		System.out.println("... done");

		Reader fr = null;
		
		switch( size ) {
		case LARGE: fr = new OhsuMed();
					break;
		default:
					break;
					
		case HUGE:	fr = new ClefIP();
					break;
		}
		
		System.out.println("reading topics ...");
		fr.readTopic( );
		if( verbose ) {
			for (Topic topic : Collection.topics ) {
				topic.print();
			}
		}
		System.out.println("... done");
		
		System.out.println("reading relevance ...");
		fr.readRelevance( );
		System.out.println("... done");
	}		

	/*
	 * 
	 */
	private static List<String> getFileNames(List<String> fileNames, Path dir) {
	    try(DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
	        for (Path path : stream) {
	            if(path.toFile().isDirectory()) {
	                getFileNames(fileNames, path);
	            } else {
	                fileNames.add(path.toAbsolutePath().toString());
	            }
	        }
	    } catch(IOException e) {
	        e.printStackTrace();
	    }
	    return fileNames;
	}	
	
	/*
	 * 
	 */
	private static void indexCorpus( CollectionSize size, IndexWriter writer ) throws IOException, SAXException, ParserConfigurationException {

		System.out.println("reading stop words ...");
		Stop.readWords();
		System.out.println("... done");
		
		Reader fr = null;
		
		switch( size ) {
		case LARGE: fr = new OhsuMed();
					break;

		case HUGE:	fr = new ClefIP();
					break;

		default:
					break;
		}

		if( size != CollectionSize.HUGE ) {

			System.out.println("reading abstracts ...");
			fr.readAbstract();
			System.out.println("... done");

			System.out.println("converting abstracts ...");
			for ( Abstract doc : Reader.abstracts) {
		
				add(writer, doc.content, doc.title, doc.ID + "", doc.index + "" );
				
				if( verbose ) {
					doc.print();
				}
			}
			System.out.println("... done");
			
		} else {

			// file walker
			String filePath = new String( "C:\\DATA\\ClefIP" );
			Path dir = FileSystems.getDefault().getPath( filePath );
			
			// 1. outer loop: directory traversal
			List<String> list = new ArrayList<String>(); 
			list = getFileNames( list, dir );
			
			String prevPath = new String( "" );

			ClefIP reader = (ClefIP) fr;
			ClefIPHandler handler = null;

			String content = null, title = null, ID = null;
			
			// 2. reading one XML file per directory
			int index = 0, count = 0;
			for( String listElement : list ) {

				String path = listElement.split("\\\\EP-[0-9]+-[A-Z,0-9]+.xml$")[0];
				
				if( !prevPath.equals(path) ) {

					// just some statistics
					count++;
					if( ( count % 1000 ) == 0 ) {
						System.out.println( "#" + index + " of " + count + " documents indexed ...");
					}
					
					// index previous file
					if( content != null && title != null && ID != null ) {
						
						Index.add(writer, content, title, ID, index + "" );
						index++;
					}
					// initialize
					content = title = ID = null;
				}
					
				// parse XML
				handler = reader.readAbstract( listElement );

				// ID
				if( handler.patentNumber != null ) {
					if( ID == null )
						ID = handler.getPatentNumber();
				}
				
				// title
				if( handler.title != null ) {
					if( title == null )
						title = handler.getTitle();
				}
				
				// content
				if( handler.content != null ) {
					if( content == null )
						content = handler.getContent();
				} else if ( handler.description != null ) {
					if( content == null ) 
						content = handler.getDescription();
				}
				
				prevPath = path;
			}
			// index previous file
			if( content != null && title != null && ID != null ) {
				
				add(writer, content, title, ID, index + "" );
				index++;
			}
		}
	}
	
	/*
	 * 
	 */
	public static void createIndex( CollectionSize size ) throws IOException, SAXException, ParserConfigurationException {

		// analyzer used for indexing and searching
		analyzer = new StandardAnalyzer(Version.LUCENE_4_9);

		switch( size ) {
		case LARGE: Index.index = FSDirectory.open(new File("c:\\temp\\large"));	
					break;

		case HUGE:	Index.index = FSDirectory.open(new File("c:\\temp\\huge"));	
					break;
					
		default:
					break;
		}			

		IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_4_9,
				new StandardAnalyzer(Version.LUCENE_4_9) );
		config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
		
		IndexWriter writer = new IndexWriter(index, config);

		System.out.println("indexing documents ...");

		indexCorpus( size, writer );
		
		System.out.println("... done");
		
		numDocs = writer.numDocs();
		System.out.println( numDocs + " documents indexed." );
		
		writer.commit();
		writer.close();
	}
	
	/*
	 * 
	 */
	public static void loadIndex( CollectionSize size ) throws IOException, ParserConfigurationException, SAXException {

		// analyzer used for indexing and searching
		analyzer = new StandardAnalyzer(Version.LUCENE_4_9);
		
		System.out.println("opening index ...");

		switch( size ) {
		case LARGE: Index.index = FSDirectory.open(new File("c:\\temp\\large"));	
					break;

		case HUGE:	Index.index = FSDirectory.open(new File("c:\\temp\\huge"));	
					break;
					
		default:
					break;
		}

		Index.__reader = DirectoryReader.open( Index.index );

		System.out.println("... done");
		
		IndexSearcher searcher = new IndexSearcher(Index.getReader());
		searcher.setSimilarity(new BM25Similarity());
		
		CollectionStatistics stats = searcher.collectionStatistics( Index.FIELD_CONTENT);
		
		avgDocLen = ( double )stats.sumTotalTermFreq() / ( double )stats.maxDoc();
		numDocs   = Index.__reader.numDocs();

		System.out.println( numDocs + " documents read." );			
		System.out.println( "Average document length: " + avgDocLen );
		
		readTopics( size );
	}

	/*
	 * 
	 */
	public static void add(IndexWriter w, String content, String title, String id, String index )
				throws IOException {
		
		org.apache.lucene.document.Document doc = new org.apache.lucene.document.Document();

		FieldType fieldType = new FieldType(TextField.TYPE_STORED);
		fieldType.setStored(true);
	    fieldType.setStoreTermVectors(true);
	    fieldType.setIndexed(true);
	    fieldType.setTokenized(true);
	    fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);

	    doc.add(new Field(Index.FIELD_CONTENT, content, fieldType));
		doc.add(new TextField(Index.FIELD_TITLE, title, Field.Store.YES));
		doc.add(new StringField(Index.FIELD_ID, id, Field.Store.YES));
		doc.add(new StringField(Index.FIELD_INDEX, index, Field.Store.YES));

		w.addDocument(doc);
	}
}
